package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"log"
	"net/http"
)

func Fetch(url string) ([]byte,error)  {
	resp,err:=http.Get(url)

	//fmt.Println(url)

	if err !=nil {
		return nil,err
	}
	defer resp.Body.Close()

	if resp.StatusCode!=http.StatusOK {
		return nil,
		 fmt.Errorf("wrong status code:%d",resp.StatusCode)
	}
	bufoRead:=bufio.NewReader(resp.Body)
	encoding:=determineEncoding(bufoRead)
	utf8Reader:=transform.NewReader(bufoRead,encoding.NewDecoder())
	all,err:=ioutil.ReadAll(utf8Reader)
	if err !=nil {
		return nil,err
	}
	//log.Printf("%s",all)
	return all,nil
}


func determineEncoding(
	r *bufio.Reader) encoding.Encoding {
	bytes,err:=r.Peek(1024)
	if err !=nil{
		log.Printf("err:%s",err)
		return unicode.UTF8
	}
	e,_,_:=charset.DetermineEncoding(bytes,"")
	return e
}

